# !pip install seaborn numpy pandas matplotlib plotly scikit-learn nltk mlxtend
# Data manipulation and analysis
import numpy as np
import pandas as pd
from datetime import datetime, timedelta

# Visualization
import seaborn as sns
import plotly.express as px
import matplotlib.pyplot as plt
import plotly.graph_objects as go
from plotly.subplots import make_subplots

# Machine Learning
import warnings
from sklearn.svm import SVC
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.metrics import silhouette_score, accuracy_score, f1_score
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, VotingClassifier

# Suppress warnings from sklearn
warnings.filterwarnings("ignore", module="sklearn")

# # Market Basket Analysis
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules

# Text Analysis
import nltk
from nltk.corpus import stopwords

# Configure plotting
plt.style.use('seaborn-v0_8')  # or 'default'
sns.set_theme()  # This sets up seaborn's default styling
pd.set_option('display.max_columns', None)

# Download required NLTK data
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')

[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\andre\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\andre\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package averaged_perceptron_tagger to
[nltk_data]     C:\Users\andre\AppData\Roaming\nltk_data...
[nltk_data]   Package averaged_perceptron_tagger is already up-to-
[nltk_data]       date!

True

# Function to load data
def load_data(file_path):
    try:
        df = pd.read_csv(file_path, 
                        encoding='ISO-8859-1',
                        dtype={'CustomerID': str, 'InvoiceNo': str})
        return df
    except FileNotFoundError:
        print(f"Error: Could not find file at {file_path}")
        print("Please ensure the data file is in the correct location")
        return None
    except Exception as e:
        print(f"Error loading data: {str(e)}")
        return None

# Try to load data from different possible locations
possible_paths = [
    'data.csv',
    '../input/data.csv',
    'customer-segmentation.csv'
]

df = None
for path in possible_paths:
    df = load_data(path)
    if df is not None:
        print(f"Successfully loaded data from {path}")
        break

if df is None:
    print("Could not load data. Please provide the correct file path.")
else:
    # Display basic information
    print("\nDataset Overview:")
    print(f"Number of rows: {len(df):,}")
    print(f"Number of columns: {len(df.columns):,}")
    print("\nColumns:", df.columns.tolist())
    
    # Display sample
    print("\nFirst few rows:")
    display(df.head())
    
    # Data quality check
    print("\nData Quality Check:")
    quality_report = pd.DataFrame({
        'dtype': df.dtypes,
        'null_count': df.isnull().sum(),
        'null_percentage': (df.isnull().sum() / len(df) * 100).round(2),
        'unique_values': df.nunique()
    })
    display(quality_report)

Successfully loaded data from data.csv

Dataset Overview:
Number of rows: 541,909
Number of columns: 8

Columns: ['InvoiceNo', 'StockCode', 'Description', 'Quantity', 'InvoiceDate', 'UnitPrice', 'CustomerID', 'Country']

First few rows:

Data Quality Check:

def preprocess_data(df):
    """Comprehensive data preprocessing function"""
    if df is None:
        return None
    
    # Create a copy to avoid modifying original data
    df_clean = df.copy()
    
    # Convert date column
    df_clean['InvoiceDate'] = pd.to_datetime(df_clean['InvoiceDate'])
    
    # Remove missing CustomerIDs
    df_clean = df_clean.dropna(subset=['CustomerID'])
    
    # Calculate total amount
    df_clean['TotalAmount'] = df_clean['Quantity'] * df_clean['UnitPrice']
    
    # Remove cancelled orders and returns (negative quantities)
    df_clean = df_clean[
        (df_clean['Quantity'] > 0) & 
        (~df_clean['InvoiceNo'].str.startswith('C', na=False))
    ]
    
    # Remove outliers based on quantity and unit price
    df_clean = df_clean[
        (df_clean['Quantity'] <= df_clean['Quantity'].quantile(0.99)) &
        (df_clean['UnitPrice'] <= df_clean['UnitPrice'].quantile(0.99))
    ]
    
    # Add derived features
    df_clean['Year'] = df_clean['InvoiceDate'].dt.year
    df_clean['Month'] = df_clean['InvoiceDate'].dt.month
    df_clean['Day'] = df_clean['InvoiceDate'].dt.day
    df_clean['DayOfWeek'] = df_clean['InvoiceDate'].dt.dayofweek
    
    return df_clean

# Preprocess the data
df_clean = preprocess_data(df)

# Display preprocessing results
if df_clean is not None:
    print("Preprocessing Results:")
    print(f"Original rows: {len(df):,}")
    print(f"Cleaned rows: {len(df_clean):,}")
    print(f"Rows removed: {len(df) - len(df_clean):,}")
    
    # Display sample of cleaned data
    display(df_clean.head())
    
    # Display summary statistics
    print("\nSummary Statistics:")
    display(df_clean.describe())

Preprocessing Results:
Original rows: 541,909
Cleaned rows: 390,294
Rows removed: 151,615

Summary Statistics:

def perform_rfm_analysis(df):
    """Perform RFM analysis on the dataset"""
    # Calculate Recency, Frequency, Monetary values
    snapshot_date = df['InvoiceDate'].max() + timedelta(days=1)
    
    rfm = df.groupby('CustomerID').agg({
        'InvoiceDate': lambda x: (snapshot_date - x.max()).days,  # Recency
        'InvoiceNo': 'count',  # Frequency
        'TotalAmount': 'sum'   # Monetary
    })
    
    # Rename columns
    rfm.columns = ['Recency', 'Frequency', 'Monetary']
    
    # Create RFM scores
    r_labels = range(4, 0, -1)
    f_labels = range(1, 5)
    m_labels = range(1, 5)
    
    r_quartiles = pd.qcut(rfm['Recency'], q=4, labels=r_labels)
    f_quartiles = pd.qcut(rfm['Frequency'], q=4, labels=f_labels)
    m_quartiles = pd.qcut(rfm['Monetary'], q=4, labels=m_labels)
    
    rfm['R'] = r_quartiles
    rfm['F'] = f_quartiles
    rfm['M'] = m_quartiles
    
    # Calculate RFM Score
    rfm['RFM_Score'] = rfm['R'].astype(str) + rfm['F'].astype(str) + rfm['M'].astype(str)
    
    # Segment customers
    def segment_customers(row):
        if row['R'] == 4 and row['F'] == 4 and row['M'] == 4:
            return 'Best Customers'
        elif row['F'] == 4 and row['M'] == 4:
            return 'Loyal Customers'
        elif row['M'] == 4:
            return 'Big Spenders'
        elif row['F'] == 4:
            return 'Frequent Shoppers'
        elif row['R'] == 4:
            return 'Recent Customers'
        elif row['R'] == 1:
            return 'Lost Customers'
        else:
            return 'Average Customers'
    
    rfm['Customer_Segment'] = rfm.apply(segment_customers, axis=1)
    
    return rfm

# Perform RFM analysis
rfm_df = perform_rfm_analysis(df_clean)

fig = make_subplots(
    rows=2, cols=2,
    specs=[[{'type': 'domain'}, {'type': 'xy'}],
           [{'type': 'xy'}, {'type': 'xy'}]],
    subplot_titles=('Customer Segments Distribution', 'Average Monetary Value by Segment',
                   'Average Recency by Segment', 'Average Frequency by Segment')
)

# Segment distribution pie chart
segment_dist = rfm_df['Customer_Segment'].value_counts()
fig.add_trace(
    go.Pie(labels=segment_dist.index, values=segment_dist.values),
    row=1, col=1
)

# Average monetary value by segment
avg_monetary = rfm_df.groupby('Customer_Segment')['Monetary'].mean().sort_values(ascending=True)
fig.add_trace(
    go.Bar(x=avg_monetary.index, y=avg_monetary.values, name='Avg Monetary Value'),
    row=1, col=2
)
fig.update_yaxes(title_text="Pounds (£)", row=1, col=2, tickangle=45)

# Average recency by segment
avg_recency = rfm_df.groupby('Customer_Segment')['Recency'].mean().sort_values(ascending=True)
fig.add_trace(
    go.Bar(x=avg_recency.index, y=avg_recency.values, name='Avg Recency'),
    row=2, col=1
)
fig.update_yaxes(title_text="Days since last visit", row=2, col=1, tickangle=45)

# Average frequency by segment
avg_frequency = rfm_df.groupby('Customer_Segment')['Frequency'].mean().sort_values(ascending=True)
fig.add_trace(
    go.Bar(x=avg_frequency.index, y=avg_frequency.values, name='Avg Frequency'),
    row=2, col=2
)
fig.update_yaxes(title_text="Visits per Year", row=2, col=2, tickangle=45)

# Update layout
fig.update_layout(
    height=800,
    showlegend=False,
    title_text="Customer Segment Analysis"
)

# Show the figure
fig.show()

def analyze_products(df):
    """Analyze product performance and patterns"""
    # Product performance metrics
    product_metrics = df.groupby('StockCode').agg({
        'Description': 'first',
        'Quantity': 'sum',
        'TotalAmount': 'sum',
        'CustomerID': 'nunique'
    }).reset_index()
    
    product_metrics.columns = ['StockCode', 'Description', 'Total_Quantity',
                             'Total_Revenue', 'Unique_Customers']
    
    # Calculate product metrics
    product_metrics['Average_Order_Value'] = (
        product_metrics['Total_Revenue'] / product_metrics['Unique_Customers']
    )
    
    return product_metrics

# Analyze products
product_metrics = analyze_products(df_clean)

# Create visualizations
fig = make_subplots(rows=2, cols=2,
                    subplot_titles=('Top Products by Revenue',
                                  'Top Products by Quantity',
                                  'Product Price Distribution',
                                  'Customer Reach Distribution'))

# Top products by revenue
top_revenue = product_metrics.nlargest(10, 'Total_Revenue')
fig.add_trace(
    go.Bar(x=top_revenue['Description'], y=top_revenue['Total_Revenue']),
    row=1, col=1
)

# Top products by quantity
top_quantity = product_metrics.nlargest(10, 'Total_Quantity')
fig.add_trace(
    go.Bar(x=top_quantity['Description'], y=top_quantity['Total_Quantity']),
    row=1, col=2
)

# Price distribution
fig.add_trace(
    go.Histogram(x=product_metrics['Average_Order_Value'], nbinsx=50),
    row=2, col=1
)

# Customer reach distribution
fig.add_trace(
    go.Histogram(x=product_metrics['Unique_Customers'], nbinsx=50),
    row=2, col=2
)

fig.update_layout(height=1000, showlegend=False,
                 title_text="Product Analysis")
fig.show()

def prepare_modeling_data(rfm_df):
    """Prepare data for modeling"""
    # Use RFM values as features
    X = rfm_df[['Recency', 'Frequency', 'Monetary']]
    
    # Use customer segments as target
    y = rfm_df['Customer_Segment']
    
    # Scale features
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)
    
    # Split data
    X_train, X_test, y_train, y_test = train_test_split(
        X_scaled, y, test_size=0.2, random_state=42
    )
    
    return X_train, X_test, y_train, y_test

def evaluate_model(model, X_train, X_test, y_train, y_test):
    """Train and evaluate a model"""
    # Train model
    model.fit(X_train, y_train)
    
    # Make predictions
    y_pred = model.predict(X_test)
    
    # Calculate metrics
    accuracy = accuracy_score(y_test, y_pred)
    f1 = f1_score(y_test, y_pred, average='weighted')
    
    # Cross-validation score
    cv_scores = cross_val_score(model, X_train, y_train, cv=5)
    
    return {
        'accuracy': accuracy,
        'f1_score': f1,
        'cv_mean': cv_scores.mean(),
        'cv_std': cv_scores.std()
    }

# Prepare data
X_train, X_test, y_train, y_test = prepare_modeling_data(rfm_df)

# Define models to compare
models = {
    'SVM': SVC(kernel='rbf', random_state=42),
    'Logistic Regression': LogisticRegression(multi_class='multinomial', random_state=42),
    'k-NN': KNeighborsClassifier(n_neighbors=5),
    'Decision Tree': DecisionTreeClassifier(random_state=42),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'AdaBoost': AdaBoostClassifier(random_state=42),
    'Gradient Boosting': GradientBoostingClassifier(random_state=42)
}

# Evaluate all models
results = {}
for name, model in models.items():
    print(f"Evaluating {name}...")
    results[name] = evaluate_model(model, X_train, X_test, y_train, y_test)

# Create voting classifier
voting_clf = VotingClassifier(
    estimators=[(name, model) for name, model in models.items()],
    voting='hard'
)
results['Voting Classifier'] = evaluate_model(
    voting_clf, X_train, X_test, y_train, y_test
)

# Display results
results_df = pd.DataFrame(results).T
display(results_df)

# Visualize model comparison
fig = make_subplots(rows=2, cols=2,
                    subplot_titles=('Model Accuracy',
                                  'F1 Scores',
                                  'Cross-validation Scores',
                                  'Model Rankings'))

# Accuracy comparison
fig.add_trace(
    go.Bar(x=list(results.keys()), 
           y=[r['accuracy'] for r in results.values()],
           name='Accuracy'),
    row=1, col=1
)

# F1 score comparison
fig.add_trace(
    go.Bar(x=list(results.keys()), 
           y=[r['f1_score'] for r in results.values()],
           name='F1 Score'),
    row=1, col=2
)

# Cross-validation scores
fig.add_trace(
    go.Bar(x=list(results.keys()),
           y=[r['cv_mean'] for r in results.values()],
           error_y=dict(
               type='data',
               array=[r['cv_std'] for r in results.values()],
               visible=True
           ),
           name='CV Score'),
    row=2, col=1
)

# Model rankings
rankings = pd.DataFrame(results).T.mean(axis=1).sort_values(ascending=True)
fig.add_trace(
    go.Bar(x=rankings.index, y=rankings.values,
           name='Overall Ranking'),
    row=2, col=2
)

fig.update_layout(height=1000, title_text="Model Comparison Results")
fig.show()

Evaluating SVM...
Evaluating Logistic Regression...
Evaluating k-NN...
Evaluating Decision Tree...
Evaluating Random Forest...
Evaluating AdaBoost...
Evaluating Gradient Boosting...

# Analyze feature importance using Random Forest
rf_model = models['Random Forest']
feature_importance = pd.DataFrame({
    'feature': ['Recency', 'Frequency', 'Monetary'],
    'importance': rf_model.feature_importances_
}).sort_values('importance', ascending=True)

# Visualize feature importance
fig = go.Figure(go.Bar(
    x=feature_importance['importance'],
    y=feature_importance['feature'],
    orientation='h'
))

fig.update_layout(
    title='Feature Importance Analysis',
    xaxis_title='Importance Score',
    yaxis_title='Feature',
    height=400
)

fig.show()

# Analyze feature interactions
fig = make_subplots(rows=1, cols=3,
                    subplot_titles=('Recency vs Frequency',
                                  'Recency vs Monetary',
                                  'Frequency vs Monetary'))

# Map predictions (string labels) to numeric values for coloring
color_map = {
    'Lost Customers': 0,
    'Average Customers': 1,
    'Big Spenders': 2,
    'Best Customers': 3,
    'Frequent Shoppers': 4,
    'Recent Customers': 5,
    'Loyal Customers': 6
}
numeric_predictions = [color_map[label] for label in rf_model.predict(X_test)]

# Plot feature interactions with color representing predictions
fig.add_trace(
    go.Scatter(
        x=X_test[:, 0],
        y=X_test[:, 1],
        mode='markers',
        marker=dict(color=numeric_predictions, colorscale='Viridis'),  # Use a colorscale
        name='Recency vs Frequency',
        showlegend=False  # Disable legend entry
    ),
    row=1, col=1
)

fig.add_trace(
    go.Scatter(
        x=X_test[:, 0],
        y=X_test[:, 2],
        mode='markers',
        marker=dict(color=numeric_predictions, colorscale='Viridis'),
        name='Recency vs Monetary',
        showlegend=False  # Disable legend entry
    ),
    row=1, col=2
)

fig.add_trace(
    go.Scatter(
        x=X_test[:, 1],
        y=X_test[:, 2],
        mode='markers',
        marker=dict(color=numeric_predictions, colorscale='Viridis'),
        name='Frequency vs Monetary',
        showlegend=False  # Disable legend entry
    ),
    row=1, col=3
)

fig.update_layout(
    height=400,
    title_text="Feature Interactions",
    coloraxis=dict(
        colorscale='Viridis',
        colorbar=dict(title='Customer Segment')  # Add a colorbar
    )
)

fig.show()

	InvoiceNo	StockCode	Description	Quantity	InvoiceDate	UnitPrice	CustomerID	Country
0	536365	85123A	WHITE HANGING HEART T-LIGHT HOLDER	6	12/1/2010 8:26	2.55	17850	United Kingdom
1	536365	71053	WHITE METAL LANTERN	6	12/1/2010 8:26	3.39	17850	United Kingdom
2	536365	84406B	CREAM CUPID HEARTS COAT HANGER	8	12/1/2010 8:26	2.75	17850	United Kingdom
3	536365	84029G	KNITTED UNION FLAG HOT WATER BOTTLE	6	12/1/2010 8:26	3.39	17850	United Kingdom
4	536365	84029E	RED WOOLLY HOTTIE WHITE HEART.	6	12/1/2010 8:26	3.39	17850	United Kingdom

	dtype	null_count	null_percentage	unique_values
InvoiceNo	object	0	0.00	25900
StockCode	object	0	0.00	4070
Description	object	1454	0.27	4223
Quantity	int64	0	0.00	722
InvoiceDate	object	0	0.00	23260
UnitPrice	float64	0	0.00	1630
CustomerID	object	135080	24.93	4372
Country	object	0	0.00	38

	InvoiceNo	StockCode	Description	Quantity	InvoiceDate	UnitPrice	CustomerID	Country	TotalAmount	Year	Month	Day	DayOfWeek
0	536365	85123A	WHITE HANGING HEART T-LIGHT HOLDER	6	2010-12-01 08:26:00	2.55	17850	United Kingdom	15.30	2010	12	1	2
1	536365	71053	WHITE METAL LANTERN	6	2010-12-01 08:26:00	3.39	17850	United Kingdom	20.34	2010	12	1	2
2	536365	84406B	CREAM CUPID HEARTS COAT HANGER	8	2010-12-01 08:26:00	2.75	17850	United Kingdom	22.00	2010	12	1	2
3	536365	84029G	KNITTED UNION FLAG HOT WATER BOTTLE	6	2010-12-01 08:26:00	3.39	17850	United Kingdom	20.34	2010	12	1	2
4	536365	84029E	RED WOOLLY HOTTIE WHITE HEART.	6	2010-12-01 08:26:00	3.39	17850	United Kingdom	20.34	2010	12	1	2

	Quantity	InvoiceDate	UnitPrice	TotalAmount	Year	Month	Day	DayOfWeek
count	390294.000000	390294	390294.000000	390294.000000	390294.000000	390294.000000	390294.000000	390294.000000
mean	9.911954	2011-07-11 01:44:35.631447296	2.726176	17.730553	2010.934155	7.616492	15.042763	2.620363
min	1.000000	2010-12-01 08:26:00	0.000000	0.000000	2010.000000	1.000000	1.000000	0.000000
25%	2.000000	2011-04-07 11:12:00	1.250000	4.550000	2011.000000	5.000000	7.000000	1.000000
50%	6.000000	2011-07-31 15:00:00	1.950000	11.250000	2011.000000	8.000000	15.000000	2.000000
75%	12.000000	2011-10-20 15:57:00	3.750000	18.720000	2011.000000	11.000000	22.000000	4.000000
max	120.000000	2011-12-09 12:50:00	14.950000	1314.000000	2011.000000	12.000000	31.000000	6.000000
std	14.582610	NaN	2.540946	30.440012	0.248012	3.417216	8.655576	1.932740

	accuracy	f1_score	cv_mean	cv_std
SVM	0.944056	0.941592	0.935608	0.007767
Logistic Regression	0.913753	0.910254	0.916673	0.009841
k-NN	0.940559	0.938812	0.931823	0.008273
Decision Tree	1.000000	1.000000	0.999417	0.000714
Random Forest	1.000000	1.000000	0.999418	0.000713
AdaBoost	0.726107	0.629002	0.713288	0.002212
Gradient Boosting	1.000000	1.000000	0.997379	0.001931
Voting Classifier	0.989510	0.989416	0.988930	0.005864

E-commerce Customer Segmentation Analysis¶

Executive Summary¶

Key Objectives:¶

Prerequisites¶

Table of Contents¶

1. Data Loading and Initial Exploration¶

Dataset Description¶

Dataset Characteristics¶

2. Data Preprocessing and Cleaning¶

Data Quality Assessment¶

3. Customer Analysis¶

RFM Analysis¶

Analysis Results¶

4. Product Analysis¶

Product Performance¶

Product Performance Analysis¶

5. Predictive Modeling¶

Model Comparison¶

Model Performance Analysis¶

5.2 Feature Importance Analysis¶

Feature Importance Analysis¶

Feature Importance:¶

6. Business Recommendations¶

Supported Business Actions¶